'''urllib_爬取豆瓣网数据案例2爬取分页
需求：
    1.爬取豆瓣网前10页数据
    2.并将数据下载到本地
'''
import urllib.request
import urllib.parse

'''
爬取分页豆瓣网影视地址 注意：分页规则 start: 0, 20, 40 , 60 limit:20每页20条；
https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&start=0&limit=20
https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&start=20&limit=20
https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&start=40&limit=20
https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&start=60&limit=20
https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&start=80&limit=20
https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&start=100&limit=20
'''

# 创建豆瓣网请求Request
def createRequest(pageNum):
    # 爬取豆瓣网影视地址
    douban_url = 'https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&'
    douban_data = {
        "start": pageNum,
        "limit": 20
    }
    # 拼接豆瓣网影视分页地址
    douban_url = douban_url + urllib.parse.urlencode(douban_data)
    print(douban_url)
    # 定义豆瓣网影视请求头
    douban_headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36"
    }
    request = urllib.request.Request(url=douban_url, headers=douban_headers)
    return request


# 请求豆瓣网返回数据
def requestDouban(request):
    response = urllib.request.urlopen(request)
    res = response.read().decode('utf-8')
    return res;


# 下载文件
def downFile(content, filePath):
    with open(filePath, 'w', encoding='utf-8') as fp:
        fp.write(content)




if __name__ == '__main__':
    startNum = int(input("请输入起始页："))
    endNum = int(input("请输入结束页："))
    for i in range(startNum, endNum + 1):
        # pageNum 计算分页
        pageNum = (i - 1) * 20
        request = createRequest(pageNum)
        content = requestDouban(request)
        logTitle = '爬取豆瓣网_第{0}页'.format(i)
        print(logTitle, content)
        # 下载数据到本地
        downFilePath = 'D:\\hxy\\test\\'+logTitle+'.json'
        downFile(content, downFilePath)

